{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "40a37596-ab92-4a8f-8c45-1e048d2e0328",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory\n",
      "chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory\n",
      "/bin/bash: line 0: cd: /mnt/amin: No such file or directory\n"
     ]
    }
   ],
   "source": [
    "pip install openai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9a22bd77-6cd4-41ec-a0b8-1140c1b36cb0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import time,json\n",
    "from cubestudio.request.model_client import Client,init\n",
    "from cubestudio.train.task import InferenceService,Project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1f1bb79c-232e-4e77-9590-5de797a453ba",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "初始化验证成功\n"
     ]
    }
   ],
   "source": [
    "# 初始化认证\n",
    "HOST = \"http://kubeflow-dashboard.infra/\"\n",
    "token = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJjdWJlLXN0dWRpbyIsInN1YiI6ImFkbWluIn0.z4XJRqUI4v39MUYDUKdIuQsP5QlRENyVkQIp6a-1fb0'\n",
    "username='admin'\n",
    "init(host=HOST,username=username,token=token)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e596af1a-852c-4a5e-9963-7a934cafa83e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 添加一个inferenceservice\n",
    "inferenceservice = Client(InferenceService).add_or_update(\n",
    "    service_type=f'vllm',\n",
    "    project=Client(Project).one(name='public'),\n",
    "    label='chatglm3-6b对话模型',\n",
    "    model_name='chatglm3-6b',\n",
    "    model_version='v202300801',\n",
    "    model_path='/mnt/admin/pipeline/example/sdk/chatglm/chatglm3-6b',\n",
    "    images='ccr.ccs.tencentyun.com/cube-studio/llm-server:20240601',\n",
    "    resource_memory='20G',\n",
    "    host=\"/v1/models\",\n",
    "    resource_cpu='10',\n",
    "    resource_gpu='1',\n",
    "    min_replicas='1',\n",
    "    max_replicas='1',\n",
    "    ports='8000',\n",
    "    volume_mount='kubeflow-user-workspace(pvc):/mnt',\n",
    "    working_dir='',\n",
    "    command='python -m vllm.entrypoints.openai.api_server --trust-remote-code --model $KUBEFLOW_MODEL_PATH --host 0.0.0.0 --port 8000 --dtype float16 --tensor-parallel-size $RESOURCE_GPU --served-model-name chatglm3-6b',\n",
    "    env='HF_ENDPOINT=https://hf-mirror.com',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2c0458bf-4a04-438f-98f8-53c300049995",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"service_type\": \"vllm\",\n",
      "    \"project\": {\n",
      "        \"created_by\": null,\n",
      "        \"changed_by\": null,\n",
      "        \"created_on\": \"2024-12-06T21:48:34\",\n",
      "        \"changed_on\": \"2024-12-06T21:48:34\",\n",
      "        \"name\": \"public\",\n",
      "        \"describe\": \"\\u516c\\u5171\\u9879\\u76ee\\u7ec4\",\n",
      "        \"type\": \"org\",\n",
      "        \"expand\": \"{\\n    \\\"cluster\\\": \\\"dev\\\",\\n    \\\"org\\\": \\\"public\\\"\\n}\",\n",
      "        \"user\": null\n",
      "    },\n",
      "    \"name\": \"chatglm3-6b-202300801\",\n",
      "    \"label\": \"chatglm3-6b\\u5bf9\\u8bdd\\u6a21\\u578b\",\n",
      "    \"model_name\": \"chatglm3-6b\",\n",
      "    \"model_version\": \"v202300801\",\n",
      "    \"images\": \"ccr.ccs.tencentyun.com/cube-studio/llm-server:20240601\",\n",
      "    \"model_path\": \"/mnt/admin/pipeline/example/sdk/chatglm/chatglm3-6b\",\n",
      "    \"volume_mount\": \"kubeflow-user-workspace(pvc):/mnt\",\n",
      "    \"sidecar\": \"\",\n",
      "    \"working_dir\": \"\",\n",
      "    \"command\": \"python -m vllm.entrypoints.openai.api_server --trust-remote-code --model $KUBEFLOW_MODEL_PATH --host 0.0.0.0 --port 8000 --dtype float16 --tensor-parallel-size $RESOURCE_GPU --served-model-name chatglm3-6b\",\n",
      "    \"env\": \"HF_ENDPOINT=https://hf-mirror.com\",\n",
      "    \"resource_memory\": \"20G\",\n",
      "    \"resource_cpu\": \"10\",\n",
      "    \"resource_gpu\": \"1\",\n",
      "    \"min_replicas\": 1,\n",
      "    \"max_replicas\": 1,\n",
      "    \"ports\": \"8000\",\n",
      "    \"inference_host_url\": \"<a target=_blank href=\\\"http://8000/v1/models\\\">8000</a>\",\n",
      "    \"hpa\": \"\",\n",
      "    \"cronhpa\": \"\",\n",
      "    \"priority\": 1,\n",
      "    \"canary\": \"\",\n",
      "    \"shadow\": \"\",\n",
      "    \"health\": \"8000:/v1/models\",\n",
      "    \"model_status\": \"offline\",\n",
      "    \"expand\": \"{\\n    \\\"public_key\\\": {\\n        \\\"alg\\\": \\\"RS256\\\",\\n        \\\"e\\\": \\\"AQAB\\\",\\n        \\\"kid\\\": \\\"chatglm3-6b-202300801\\\",\\n        \\\"kty\\\": \\\"RSA\\\",\\n        \\\"n\\\": \\\"zZQl59SX4W_MM95sAULV3rgjC4js6f11XhEAIi9MKrsk8LY9Luom8vcQtQPgXAC-NdwzkE9Bg_JQsDmuaXxFfCbJwV0o9TekX0onrfcIvTS2n7pguLordN2hUDMZNOJVzBdgScUeCbu3l2oI3oC_sb0OKx3AF7Omm8z-k3yhWF8NtQValrahonk7Vpx8hFU9LfvaYLW5r8l3EJJ-u5THSWOStAEAQeVWWYbMdD83fF3KGDyzcbAFaEAE7u2R6cwu3QQap3aUTq3KXMx7OEn-GT7CIqY8lg82oPf8FpkbK6xRK8SN8kuoSfAG4NdXPJ4iY8_ecU3nDN1mvGWIzNd0TQ\\\"\\n    },\\n    \\\"private_key\\\": {\\n        \\\"alg\\\": \\\"RS256\\\",\\n        \\\"d\\\": \\\"KUNzylFDPRZGHCEod6ZVLFO0gRO2Tg84WC28Y6AvvyH6O7CwnjKUp88MlPDWv1DM3R26Ydxur4_bP0wMpJKzlcRpWb0bxACrpw7I4SPFJskmYPA1is6fe2EjCGiBRjqp8wG8K4J70YNSyfsYFaUtFP14ExJxJLDEH-CpNqxc7FIniJ4UEni-3yO0cCgNAm6nurrZqHhMIjjNI5yAgv2nNyO_vMxtkYkkT0yzOSeuL5SW0dH8iWleuI3Mv2zQ6B9Xra7aiKmdIqEONFIn4VmQ-SeqjNy7skt-ogdYh1luTMPwBSNjcBAL1Xwc7SzGFaFgLujQg3o2r6u0IZjZ8v2Pdw\\\",\\n        \\\"dp\\\": \\\"Iz0ncnUrqI7NIOnUK42v-CjT0CgcXWN27eCB0gY3EtQa_YyTWL_CeNWxKt2LaYxUc___8mXADdylc5IcCnkxQHnJ9o_PqQ3Cw_dudiebIiytTGikJU91WGRBISQXIsB6sbJGPP9eMp9p4fhr8F3kiZb67370XvKwxz9cL5j-x-M\\\",\\n        \\\"dq\\\": \\\"F8CI-wj1pC70RA4jaTq17H_MJ9clpVDXrwxGmzpsbvr5VpaQCmGrE0LWhLnAHJXCpbzg1OgM-2u3_bWOjHjW3L9uYQVwPIki09O6gzGWAIeT-Mn3XyIv9KPwszurANF6IBeQIOSHwfnXJJrO4R7wfEv-oSE6a8xOeNWWQ6fkmKE\\\",\\n        \\\"e\\\": \\\"AQAB\\\",\\n        \\\"kid\\\": \\\"chatglm3-6b-202300801\\\",\\n        \\\"kty\\\": \\\"RSA\\\",\\n        \\\"n\\\": \\\"zZQl59SX4W_MM95sAULV3rgjC4js6f11XhEAIi9MKrsk8LY9Luom8vcQtQPgXAC-NdwzkE9Bg_JQsDmuaXxFfCbJwV0o9TekX0onrfcIvTS2n7pguLordN2hUDMZNOJVzBdgScUeCbu3l2oI3oC_sb0OKx3AF7Omm8z-k3yhWF8NtQValrahonk7Vpx8hFU9LfvaYLW5r8l3EJJ-u5THSWOStAEAQeVWWYbMdD83fF3KGDyzcbAFaEAE7u2R6cwu3QQap3aUTq3KXMx7OEn-GT7CIqY8lg82oPf8FpkbK6xRK8SN8kuoSfAG4NdXPJ4iY8_ecU3nDN1mvGWIzNd0TQ\\\",\\n        \\\"p\\\": \\\"-nSfWOK2q872W8Es4AqJXZw4VIHxqRkUHBSJN9TxV6374avK9Fe1Y-wtb255vfkdaO2JTDNDM-hwKtp7VpIUbJTDytscCV4jautwkPqRBr23PH5HvM2g_egcW1FTHgA0661D6kdT7SqEzLsQCn1Yk5K9k7jseuhBWgnP8Qp6H_s\\\",\\n        \\\"q\\\": \\\"0iEzRKDr_w2P3LJMAQp54_-_BRcitiItVda6CfzygIhOA93ywz0CYctCDvAh8mFJYdmF3gazQ8Gjb5qv5bchXjaR_ELyPs2po-O-cTaA5wkAkMrs9mC6kY0tlkSAJ7A0lMrtfjnNW0lMzTL9GEC_JOx4_TsCG-DWtX8jtL1h4lc\\\",\\n        \\\"qi\\\": \\\"Elo-JJb6YQaEs_PkZQYzA26m1RZ0Z9JFyoNl4-M89B-u8QMx7yN1a8yq0BrGRqxnOtvO_Bdo_Ox3-ZCX5AZhyVTu25RTdmzRF2AZW_SBJras0JQO4hcuUnQGrBx-pLsvlEKh5drSGo7OPWPFRdPpqusL8Y1OX63JqLyY43FvfqY\\\"\\n    }\\n}\",\n",
      "    \"metrics\": \"\",\n",
      "    \"deploy_history\": \"\",\n",
      "    \"host\": \"8000:/v1/models\",\n",
      "    \"inference_config\": \"\"\n",
      "}\n",
      "推理服务发布成功，滚动升级中，注意接收升级消息\n"
     ]
    }
   ],
   "source": [
    "print(json.dumps(inferenceservice.to_dict(), indent=4))\n",
    "if inferenceservice.model_status!='online':\n",
    "    inferenceservice.deploy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "7b3cd4e6-db26-4084-b2f1-c71451078edb",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 好的，请问您想了解哪方面的北京呢？例如历史、文化、美食等。\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "import json\n",
    "\n",
    "url = \"http://chatglm3-6b-202300801.service:8000/v1/chat/completions\"\n",
    "\n",
    "payload = json.dumps({\n",
    "  \"model\": \"chatglm3-6b\",\n",
    "  \"messages\": [\n",
    "    {\n",
    "      \"role\": \"user\",\n",
    "      \"content\": \"介绍一下北京。\"\n",
    "    }\n",
    "  ],\n",
    "  \"temperature\": 0,\n",
    "  \"stop\": \"<|im_end|>\"\n",
    "})\n",
    "\n",
    "response = requests.request(\"POST\", url, data=payload)\n",
    "content = response.json()['choices'][0]['message']['content']\n",
    "print(content)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4c5fd24b-09dc-4511-a1c5-75ceec2fa7d5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "清理服务完成\n"
     ]
    }
   ],
   "source": [
    "inferenceservice.clear()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ab113e7-8f4c-443f-885a-4a3909a668e9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "cube-studio",
   "language": "python",
   "name": "cube-studio"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
